{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "06482678",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-21 16:10:38.550405: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2022-04-21 16:10:38.550463: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "Using custom data configuration default-c7b337de7921aa29\n",
      "Reusing dataset csv (/home/zshengbo/.cache/huggingface/datasets/csv/default-c7b337de7921aa29/0.0.0/9144e0a4e8435090117cea53e6c7537173ef2304525df4a077c435d8ee7828ff)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e2155fa359074326b17d8772cabd7571",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "dataset = load_dataset('csv', data_files='/home/zshengbo/py/ChampSim-master2/111.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4b78a35d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "#从API中加载模型\n",
    "from transformers import DistilBertTokenizer\n",
    "tokenizer = DistilBertTokenizer.from_pretrained(\"distilbert-base-uncased\")\n",
    "\n",
    "from transformers import DistilBertForSequenceClassification\n",
    "model = DistilBertForSequenceClassification.from_pretrained(\"distilbert-base-uncased\", num_labels=2,output_attentions = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "eaa9436b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_function(examples):\n",
    "    return tokenizer(examples[\"text\"], truncation=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fbcf16c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import DataCollatorWithPadding\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8f83806b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from datasets import load_metric\n",
    "\n",
    "metric = load_metric(\"./accuracy.py\")\n",
    "\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits[0], axis=-1)\n",
    "    return metric.compute(predictions=predictions, references=labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30a074ec",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6b50a473",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['Unnamed: 0', 'PC', 'Deltas'],\n",
       "        num_rows: 208902\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9bde6490",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PreTrainedTokenizer(name_or_path='', vocab_size=50000, model_max_len=1000000000000000019884624838656, is_fast=False, padding_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "46d4a4e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizer\n",
    "import tokenizers\n",
    "# 创建分词器\n",
    "bwpt = tokenizers.BertWordPieceTokenizer()\n",
    "filepath = \"/home/zshengbo/py/ChampSim-master2/111.csv\" # 语料文件\n",
    "#训练分词器\n",
    "bwpt.train(\n",
    "    files=[filepath],\n",
    "    vocab_size=50000, # 这里预设定的词语大小不是很重要\n",
    "    min_frequency=1,\n",
    "    limit_alphabet=1000\n",
    ")\n",
    "# 保存训练后的模型词表\n",
    "bwpt.save_model('/home/zshengbo/py/bs/')\n",
    "#output： ['./pretrained_models/vocab.txt']\n",
    "\n",
    "# 加载刚刚训练的tokenizer\n",
    "tokenizer=BertTokenizer(vocab_file='/home/zshengbo/py/bs/vocab.txt')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "df00cd0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Embedding(50000, 768)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import (\n",
    "    CONFIG_MAPPING,MODEL_FOR_MASKED_LM_MAPPING, AutoConfig,\n",
    "    AutoModelForMaskedLM,\n",
    "    AutoTokenizer,DataCollatorForLanguageModeling,HfArgumentParser,Trainer,TrainingArguments,set_seed,\n",
    ")\n",
    "# 自己修改部分配置参数\n",
    "config_kwargs = {\n",
    "    \"cache_dir\": None,\n",
    "    \"revision\": 'main',\n",
    "    \"use_auth_token\": None,\n",
    "#      \"hidden_size\": 512,\n",
    "#     \"num_attention_heads\": 4,\n",
    "    \"hidden_dropout_prob\": 0.2,\n",
    "#     \"vocab_size\": 863 # 自己设置词汇大小\n",
    "}\n",
    "# 将模型的配置参数载入\n",
    "config = AutoConfig.from_pretrained('bert-base-cased', **config_kwargs)\n",
    "# 载入预训练模型\n",
    "model = AutoModelForMaskedLM.from_pretrained(\n",
    "            'bert-base-cased',\n",
    "            from_tf=bool(\".ckpt\" in 'roberta-base'), # 支持tf的权重\n",
    "            config=config,\n",
    "            cache_dir=None, \n",
    "            #revision='main',\n",
    "            #use_auth_token=None,\n",
    "        )\n",
    "model.resize_token_embeddings(len(tokenizer))\n",
    "#output:Embedding(863, 768, padding_idx=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "72098d54",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default-227967aa841fe6b3\n",
      "Reusing dataset text (/home/zshengbo/.cache/huggingface/datasets/text/default-227967aa841fe6b3/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7eb77b5ebc634e5a94cd362bdbe8eab7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "576e08709687464ba37dac148deeafb7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/209 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b9ebc99e6394e5e92b2e2db4d881ad0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/209 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset,Dataset\n",
    "# data_files接收字典或者list，值为语料路径\n",
    "datasets=load_dataset('text',data_files={'train':'/home/zshengbo/py/ChampSim-master2/111.csv',\"validation\":\"/home/zshengbo/py/ChampSim-master2/111.csv\"})\n",
    "column_names = datasets[\"train\"].column_names\n",
    "text_column_name = \"text\" if \"text\" in column_names else column_names[0]\n",
    "# 将我们刚刚加载好的datasets ，通过tokenizer做映射，得到input_id，也就是实际输入模型的东西。\n",
    "def tokenize_function(examples):\n",
    "    # Remove empty lines\n",
    "    examples[\"text\"] = [line for line in examples[\"text\"] if len(line) > 0 and not line.isspace()]\n",
    "    return tokenizer(\n",
    "        examples[\"text\"], \n",
    "        padding=\"max_length\", # 进行填充\n",
    "        truncation=True, # 进行截断\n",
    "        max_length=100, # 设置句子的长度\n",
    "        # We use this option because DataCollatorForLanguageModeling (see below) is more efficient when it\n",
    "        # receives the `special_tokens_mask`.\n",
    "        return_special_tokens_mask=True,\n",
    "    )\n",
    "tokenized_datasets = datasets.map(\n",
    "    tokenize_function,\n",
    "    batched=True,\n",
    "    num_proc=None,\n",
    "    remove_columns=[text_column_name],\n",
    "    load_from_cache_file='/home/zshengbo/py/ChampSim-master2/1111.csv',\n",
    ")\n",
    "# 得到训练集和验证集\n",
    "train_dataset = tokenized_datasets[\"train\"]\n",
    "eval_dataset = tokenized_datasets[\"validation\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "52afb9ce",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'model.modeling_nezha'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_171557/1672683578.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcsv\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtransformers\u001b[0m \u001b[0;32mimport\u001b[0m  \u001b[0mBertTokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mWEIGHTS_NAME\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mTrainingArguments\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodeling_nezha\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mNeZhaForSequenceClassification\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mNeZhaForMaskedLM\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfiguration_nezha\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mNeZhaConfig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtokenizers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'model.modeling_nezha'"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import csv\n",
    "from transformers import  BertTokenizer, WEIGHTS_NAME,TrainingArguments\n",
    "from model.modeling_nezha import NeZhaForSequenceClassification,NeZhaForMaskedLM\n",
    "from model.configuration_nezha import NeZhaConfig\n",
    "import tokenizers\n",
    "import torch\n",
    "from datasets import load_dataset,Dataset \n",
    "from transformers import (\n",
    "    CONFIG_MAPPING,\n",
    "    MODEL_FOR_MASKED_LM_MAPPING,\n",
    "    AutoConfig,\n",
    "    AutoModelForMaskedLM,\n",
    "    AutoTokenizer,\n",
    "    DataCollatorForLanguageModeling,\n",
    "    HfArgumentParser,\n",
    "    Trainer,\n",
    "    TrainingArguments,\n",
    "    set_seed,\n",
    "    LineByLineTextDataset\n",
    ")\n",
    "## 制作自己的tokenizer\n",
    "bwpt = tokenizers.BertWordPieceTokenizer()\n",
    "filepath = \"../excel2txt.txt\" # 和本文第一部分的语料格式一致\n",
    "bwpt.train(\n",
    "    files=[filepath],\n",
    "    vocab_size=50000,\n",
    "    min_frequency=1,\n",
    "    limit_alphabet=1000\n",
    ")\n",
    "bwpt.save_model('./pretrained_models/') # 得到vocab.txt\n",
    "\n",
    "## 加载tokenizer和模型\n",
    "model_path='../tmp/nezha/'\n",
    "token_path='./pretrained_models/vocab.txt'\n",
    "tokenizer =  BertTokenizer.from_pretrained(token_path, do_lower_case=True)\n",
    "config=NeZhaConfig.from_pretrained(model_path)\n",
    "model=NeZhaForMaskedLM.from_pretrained(model_path, config=config)\n",
    "model.resize_token_embeddings(len(tokenizer))\n",
    "\n",
    "# 通过LineByLineTextDataset接口 加载数据 #长度设置为128, # 这里file_path于本文第一部分的语料格式一致\n",
    "train_dataset=LineByLineTextDataset(tokenizer=tokenizer,file_path='../tmp/all_data_txt.txt',block_size=128) \n",
    "# MLM模型的数据DataCollator\n",
    "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=True, mlm_probability=0.15)\n",
    "# 训练参数\n",
    "pretrain_batch_size=64\n",
    "num_train_epochs=300\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./outputs/', overwrite_output_dir=True, num_train_epochs=num_train_epochs, learning_rate=6e-5,\n",
    "    per_device_train_batch_size=pretrain_batch_size,save_total_limit=10)# save_steps=10000\n",
    "# 通过Trainer接口训练模型\n",
    "trainer = Trainer(\n",
    "    model=model, args=training_args, data_collator=data_collator, train_dataset=train_dataset)\n",
    "\n",
    "# 开始训练\n",
    "trainer.train(True)\n",
    "trainer.save_model('./outputs/')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ba30bc6c",
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (2603710958.py, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"/tmp/ipykernel_171557/2603710958.py\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    gh repo clone lonePatient/NeZha_Chinese_PyTorch\u001b[0m\n\u001b[0m          ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "gh repo clone lonePatient/NeZha_Chinese_PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6461e5e5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
